# for each file, built a database

require "dbm"

# STRING(__LocalTimeStamp),
# UINT64(SessionID),
# INT32(SessionStartTime),
# IP(ClientIP),
# IP(ServerIP),
# STRING(L4Proto),
# INT32(ClientPort),
# INT32(ServerPort),
# STRING(ClientPayloadHEX),
# STRING(ServerPayloadHEX),
# STRING(Protocol),
# STRING(ClientPayload),
# STRING(ServerPayload)

LOCAL_TIME_STAMP 	= 1
SESSION_ID 			= 2
SESSION_START_TIME 	= 3
CLIENT_IP			= 4
SERVER_IP			= 5
L4_PROTO			= 6
CLIENT_PORT			= 7
SERVER_PORT			= 8
CLIENT_PAYLOAD_HEX 	= 9
SERVER_PAYLOAD_HEX 	= 10
PROTOCOL			= 11
CLIENT_PAYLOAD 		= 12
SERVER_PAYLOAD 		= 13


DATA_DIR = "./Jan_6/"
DB_DIR = "./db_files/"

begin
  file_list = DBM.open("db_file_list")
  
  begin
    File.delete "db_raw_*"
  rescue Exception => e
    puts "no raw db to delete"
  end

  begin
    file_list.each_value do |file_name|  
      data_file = File.open(DATA_DIR + file_name)
      db_file = DBM.open("db_raw_" + file_name)

      puts data_file.path
      counter = 0

      begin
        while (line = data_file.readline)
          # puts "to read line:"
          if line[0,1] != '#'
            # puts "in if: " + counter.to_s

            if line[-1,1] == "|"
              line = line + "NULL"
            end

            fields = line.split('|')
            # puts "split"

            fields.delete_at CLIENT_PAYLOAD_HEX
            fields.delete_at CLIENT_PAYLOAD_HEX

            # puts "db"
            db_file[counter.to_s] = fields.join("|")

            if counter % 100000 == 0          
              puts counter
              # puts db_file[counter.to_s]
            end

            counter = counter + 1
          end
        end
      rescue Exception => re
        puts "end of file"
        data_file.close
      end
      puts "database size : " + db_file.size.to_s
    end
  rescue Exception => ie
    puts ie.to_s
  end

  file_list.close
rescue Exception => e
  puts e.to_s
end